# -*- coding: utf-8 -*-
#/usr/bin/env python3

import json
import os
import re
import sys
import time
import requests
import pymysql

import threading


sem=threading.Semaphore(8) #限制线程的最大数量为8个
lock = threading.Lock()
num = 0
from bs4 import BeautifulSoup



#~~~~~~~~~~~~以下是无需修改的参数~~~~~~~~~~~~~~~~·
requests.packages.urllib3.disable_warnings() #为了避免弹出一万个warning，which is caused by 非验证的get请求

leetcode_url = 'https://leetcode-cn.com/'
leetcode_request_url='https://leetcode-cn.com/graphql/'
sign_in_url = leetcode_url + 'accounts/login/'
submissions_url = leetcode_url + 'submissions/'
config_path = "./config.json"
visited = []
ccc={}
with open(config_path, "r") as f: #读取用户名，密码，本地存储目录
    config = json.loads(f.read())   
    USERNAME = config['username'] #leetcode用户名
    PASSWORD = config['password'] #leetcode密码
    OUTPUT_DIR = config['outputDir'] #本地输出路径
    DBHOST = config['dbhost'] #数据库地址
    DBPORT = config['dbport'] #数据库端口
    DBUSER = config['dbuser'] #数据库用户名
    DBPASS = config['dbpass'] #数据库密码
    DBHOUSE = config['dbhouse'] #数据库名
    TABLENAME = config['tablename'] #数据库表名
    CRAWLERURL = config['crawlerurl'] #原始解析数据（默认阿里企业题）

FILE_FORMAT = {"C++": ".cpp", "Python3": ".py", "Python": ".py", "MySQL": ".sql", "Go": ".go", "Java": ".java",
                "C": ".c", "JavaScript": ".js", "PHP": ".php", "C#": ".cs", "Ruby": ".rb", "Swift": ".swift",
                "Scala": ".scl", "Kotlin": ".kt", "Rust": ".rs"}

START_NUM = 0
question_num=0
SLEEP_TIME = 5


def login(username, password): # 本函数修改自https://gist.github.com/fyears/487fc702ba814f0da367a17a2379e8ba，感谢@fyears
    client = requests.session()
    client.encoding = "utf-8"

    while True:
        try:
            client.get(sign_in_url, verify=False)
            login_data = {
                'login': username, 
                'password': password
            }
            result = client.post(sign_in_url, data = login_data, headers = dict(Referer = sign_in_url))
            if result.ok:
                print ("Login successfully!")
                break
        except:
            print ("Login failed! Wait till next round!")
            time.sleep(SLEEP_TIME)
    return client

def work(submission,client,db,type):
    global num #全局化 否则不能进行运算
    try:
        problem_title = submission['translatedTitle']
        if problem_title is None:
            problem_title = submission['title']
        problem_id = submission['questionId']
        ccc[problem_id] = submission

        #加入内存set，用于比较处理过的题目
        if type == 0:
          visited.append(submission)
        #lock.acquire()
        #num +=1
        #print("当前开始爬取第 {} 题,题目名字是<<{}>> :".format(problem_id, problem_title))
        #lock.release()
        submission_language = "java"
        # 判断数据库是否已经有了
        cur = db.cursor()
        #id = str(num)
        id = problem_id
        cur.execute("select COUNT(1) FROM {} where id = ".format(TABLENAME) + id)
        # 使用fetchall()方法获取查询结果 (接收全部的返回结果)
        data = cur.fetchone()
        if(data[0] == 1):
           return
        #full_path = generatePath(id, problem_title, submission_language)
        slug = slugString(submission, client)
        content = downloadcondent(submission['titleSlug'], client)
        code = downloadCode(slug, client)
        # 写入mysql
        query = "REPLACE INTO {} (id, title,content, detail) values (%s,%s,%s,%s)".format(TABLENAME)
        values = (id, problem_title, content, content+code)
        cur.execute(query, values)
        print("题号是 {} 爬取成功，已保存数据库！".format(id))
    # 改这里实现数据写本地
    #   with open(full_path, "w",encoding='utf-8') as f: # 开始写到本地
    #     f.write(content+code)
    #     print ("Writing ends!", full_path)

    except FileNotFoundError as e:
        print("Output directory doesn't exist")

    except Exception as e:
        print(e, " 发生未知异常，请留意！.")
    finally:
        if type == 0:
          db.close()
        sem.release()



def scraping(client):
    question_num = START_NUM
    visited = set()
    while True:
        html = client.get(CRAWLERURL, verify=False)
        html = json.loads(html.text)
        #多线程跑任务，使用信号量控制线程数，防止线程太多
        for submission in html:
                sem.acquire()
                db = initmysql()
                th = threading.Thread(target=work, args=(submission, client, db,0))
                #防止主线程销毁，子线程也销毁
                th.setDaemon(True)
                th.start()
        return


# 提交到git
def gitPush():
    today = time.strftime('%Y-%m-%d',time.localtime(time.time()))
    os.chdir(OUTPUT_DIR)
    instructions = ["git add .","git status", "git commit -m \""+ today + "\"", "git push -u origin master"]
    for ins in instructions:
        os.system(ins)
        print ("~~~~~~~~~~~~~" + ins + " finished! ~~~~~~~~")



REQUEST_FORMAT = {"headers": {
        'Connection': 'keep-alive',
        'Content-Type': 'application/json',
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.152 Safari/537.36'}}





#第1步请求
def slugString(submission, client):
    titleSlug =  submission['titleSlug']
    #param = {'operationName': "questionSolutionVideoArticles", "variables": {"first": 10,"questionSlug": 'valid-sudoku',"skip":0,"tagSlugs":[],"userInput":""},
             #   'query': "query questionSolutionVideoArticles($questionSlug: String!, $orderBy: SolutionArticleOrderBy, $userInput: String, $tagSlugs: [String!], $skip: Int, $first: Int) {   questionSolutionVideoArticles(questionSlug: $questionSlug, orderBy: $orderBy, userInput: $userInput, tagSlugs: $tagSlugs, skip: $skip, first: $first) {     totalNum     edges {       node {         title         slug         chargeType         isEditorsPick         byLeetcode         videosInfo {           videoId           coverUrl           duration           __typename         }         hitCount         __typename       }       __typename     }     __typename   } } "
            #       }
    param = {'operationName': "questionSolutionArticles",
             "variables": {"first": 10, "questionSlug": titleSlug, "skip": 0, "tagSlugs": [], "userInput": ""},
             'query': "query questionSolutionArticles($questionSlug: String!, $skip: Int, $first: Int, $orderBy: SolutionArticleOrderBy, $userInput: String, $tagSlugs: [String!]) {   questionSolutionArticles(questionSlug: $questionSlug, skip: $skip, first: $first, orderBy: $orderBy, userInput: $userInput, tagSlugs: $tagSlugs) {     totalNum     edges {       node {         ...solutionArticle         __typename       }       __typename     }     __typename   } }  fragment solutionArticle on SolutionArticleNode {   rewardEnabled   canEditReward   uuid   title   slug   sunk   chargeType   status   identifier   canEdit   canSee   reactionType   reactionsV2 {     count     reactionType     __typename   }   tags {     name     nameTranslated     slug     tagType     __typename   }   createdAt   thumbnail   author {     username     profile {       userAvatar       userSlug       realName       __typename     }     __typename   }   summary   topic {     id     commentCount     viewCount     __typename   }   byLeetcode   isMyFavorite   isMostPopular   isEditorsPick   hitCount   videosInfo {     videoId     coverUrl     duration     __typename   }   __typename } "
             }
    try:
        param_json = json.dumps(param).encode("utf-8")
        response = client.post(leetcode_request_url, data=param_json, headers=REQUEST_FORMAT["headers"])
        submission_details = response.json()["data"]["questionSolutionArticles"]["edges"]
        slug = submission_details[00]['node']['slug']
    except Exception as e:
        print(e, "题目是{}获取slug失败，开始重新尝试".format(titleSlug))
        time.sleep(5)
        slug = slugString(submission, client)
    finally:
        return slug

#第2步请求
def downloadcondent(titleSlug, client):
    param = {'operationName': "questionData", "variables": {"titleSlug": titleSlug},
                'query': "query questionData($titleSlug: String!) {   question(titleSlug: $titleSlug) {     questionId     questionFrontendId     categoryTitle     boundTopicId     title     titleSlug     content     translatedTitle     translatedContent     isPaidOnly     difficulty     likes     dislikes     isLiked     similarQuestions     contributors {       username       profileUrl       avatarUrl       __typename     }     langToValidPlayground     topicTags {       name       slug       translatedName       __typename     }     companyTagStats     codeSnippets {       lang       langSlug       code       __typename     }     stats     hints     solution {       id       canSeeDetail       __typename     }     status     sampleTestCase     metaData     judgerAvailable     judgeType     mysqlSchemas     enableRunCode     envInfo     book {       id       bookName       pressName       source       shortDescription       fullDescription       bookImgUrl       pressImgUrl       productUrl       __typename     }     isSubscribed     isDailyQuestion     dailyRecordStatus     editorType     ugcQuestionId     style     exampleTestcases     __typename   } } "
                    }
    try:
        param_json = json.dumps(param).encode("utf-8")
        response = client.post(leetcode_request_url, data=param_json, headers=REQUEST_FORMAT["headers"])
        submission_details = response.json()["data"]["question"]
        content = submission_details["translatedContent"]
        if content is None:
            content = submission_details['content']
    except Exception as e:
        print(e, "获取content失败，开始重新尝试")
        time.sleep(5)
        content = downloadcondent(titleSlug, client)
    finally:
        return content

#第3步请求
def downloadCode(slug, client):
    param = {'operationName': "solutionDetailArticle", "variables": {"slug": slug,"orderBy": "DEFAULT"},
                'query': "query solutionDetailArticle($slug: String!, $orderBy: SolutionArticleOrderBy!) {   solutionArticle(slug: $slug, orderBy: $orderBy) {     ...solutionArticle     content     question {       questionTitleSlug       __typename     }     position     next {       slug       title       __typename     }     prev {       slug       title       __typename     }     __typename   } }  fragment solutionArticle on SolutionArticleNode {   rewardEnabled   canEditReward   uuid   title   slug   sunk   chargeType   status   identifier   canEdit   canSee   reactionType   reactionsV2 {     count     reactionType     __typename   }   tags {     name     nameTranslated     slug     tagType     __typename   }   createdAt   thumbnail   author {     username     profile {       userAvatar       userSlug       realName       __typename     }     __typename   }   summary   topic {     id     commentCount     viewCount     __typename   }   byLeetcode   isMyFavorite   isMostPopular   isEditorsPick   hitCount   videosInfo {     videoId     coverUrl     duration     __typename   }   __typename}"   }
    try:
        param_json = json.dumps(param).encode("utf-8")
        response = client.post(leetcode_request_url, data=param_json, headers=REQUEST_FORMAT["headers"])
        submission_details = response.json()["data"]["solutionArticle"]
        code = submission_details["content"]
    except Exception as e:
        print(e, "获取code失败，开始重新尝试")
        time.sleep(5)
        code = submission_details["content"]
    finally:
        return code

#保存到本地
def generatePath(problem_id, problem_title, submission_language):
    if problem_id[0].isdigit(): # 如果题目是传统的数字题号
        problem_id = int(problem_id)
        newpath = OUTPUT_DIR + "/" + '{:0=4}'.format(problem_id) + "." + problem_title #存放的文件夹名
        filename = '{:0=4}'.format(problem_id) + "-" + problem_title + '.txt' #存放的文件名
    else: # 如果题目是新的面试题
        newpath = OUTPUT_DIR + "/" + problem_id + "." + problem_title
        filename = problem_id + "-" + problem_title + FILE_FORMAT[submission_language] #存放的文件名
    if not os.path.exists(newpath):
        os.mkdir(newpath)
    
    full_path = os.path.join(newpath, filename) #把文件夹和文件组合成新的地址
    return full_path



def initmysql():
    db = pymysql.connect(host=DBHOST, port=DBPORT, user=DBUSER, password=DBPASS, db=DBHOUSE)
    db.autocommit(1)
    return db

totalnum = 0
def check():
    global totalnum
    db = initmysql()
    cur = db.cursor()
    cur.execute("select id from {}".format(TABLENAME))
    data = cur.fetchall()
    for f in data:
        question_id = str(f[0])
        visited.remove(ccc[question_id])
    totalnum = len(data)

    if len(visited) == 0:
        return
    print("以下题目没有爬取成功 \n")
    for ff in visited:
        print(ff['questionId'])
    print("开启重试！")
    # 重试时候必须慢点请求才行
    client = login(USERNAME, PASSWORD)
    db = initmysql()
    for ff in visited:
        work(ff,client,db,1)
    print("共重试{}道题！".format(len(visited)))
    totalnum = totalnum + len(visited)
    db.close()

def main():
    starttime = time.time()
    print('开始登陆leetcode...')
    client = login(USERNAME, PASSWORD)
    print('开始多线程爬题...')
    scraping(client)
    print('爬题结束')
    client.close()
    check()
    print("共爬取{}题，耗时{}s".format(totalnum,round(time.time() - starttime)))



if __name__ == '__main__':
    main()
